#!/bin/env python3

import os
import sys
import time
import datetime
import subprocess
import logging
from copy import deepcopy

def get_logger():
    """
    get logger object
    return: logger object
    """
    #log_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../log")
    log_path = "log"
    if not os.path.exists(log_path):
        os.mkdir(log_path)
    elif not os.path.isdir(log_path):
        os.remove(log_path)
        os.mkdir(log_path)
    logfile = os.path.join(log_path, "openeuler-obs.log")
    logger = logging.getLogger("")
    logger.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - '\
            '%(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')
    file_handler = logging.FileHandler(logfile, mode='w')
    file_handler.setLevel(logging.DEBUG)
    file_handler.setFormatter(formatter)
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.DEBUG)
    stream_handler.setFormatter(formatter)
    logger.addHandler(stream_handler)
    logger.addHandler(file_handler)
    return logger

def download_upload_build_log(proj, arch, pkgname, log_name, ip, pwd, flag):
    """
    download and upload pkg build log
    return
    """
    file_path = os.path.join(proj, arch, pkgname)
    log_file = os.path.join(file_path, "%s.log" % log_name)
    if not os.path.exists(file_path):
        os.system("mkdir -p ./%s" % file_path)
    cmd = "osc remotebuildlog %s %s standard_%s %s 2>/dev/null > %s" % (proj, pkgname, arch, arch, log_file)
    count = 3
    down_flag = 0
    while count:
        res = exec_command(cmd, 8)
        if res == "TIME_OUT":
            count = count - 1
        else:
            down_flag = 1
            break
    if flag == "all_status":
        server_log_path = "/srv/www/builds/obs_build_log_all"
        server_log = os.path.join(server_log_path, log_file)
    elif flag == "failed_status":
        server_log_path = "/srv/www/builds/obs_build_log"
        error_type = classify_pkg_error_type(log_file)
        new_name = log_name + "_" + error_type + ".log"
        server_log = os.path.join(server_log_path, file_path, new_name)
    else:
        log.info("error flag")
    if down_flag == 1:
        log.info("Download %s %s %s build log success." %(proj, arch, pkgname))
        cmd1 = "sshpass -p %s ssh -o StrictHostKeyChecking=no root@%s 'if [ ! -d \"%s/%s\" ];then cd %s && mkdir -p %s;fi'"% (pwd, ip, server_log_path, file_path, server_log_path, file_path)
        res = os.system(cmd1)
        cmd2 = "sshpass -p %s scp -o StrictHostKeyChecking=no ./%s root@%s:%s" % (pwd, log_file, ip, server_log)
        res = os.system(cmd2)
        if res == 0:
            log.info("upload %s log file success." % log_file)
        else:
            log.info("upload %s log file failed." % log_file)
    else:
        log.info("Download %s %s %s build log failed." %(proj, arch, pkgname))

def exec_command(cmd, timeout=None):
    """
    return after exec command results
    """
    if timeout:
        end_time = datetime.datetime.now() + datetime.timedelta(seconds=timeout)
    sub = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=4096, shell=True)
    while True:
        if sub.poll() is not None:
            break
        time.sleep(0.1)
        if timeout:
            if end_time <= datetime.datetime.now():
                sub.kill()
                return "TIME_OUT"
    res = sub.stdout.read()
    res2 = str(res).replace("b'","").replace("'","").split("\\n")
    results = filter(None, res2)
    return list(results)

def get_last_build_time(proj, arch, pkg):
    """
    get pkg last build time
    return build time
    """
    cmd = "osc api -X GET \"build/%s/standard_%s/%s/_jobhistory?package=%s&limit=1\" 2>/dev/null" % (proj, arch, arch, pkg)
    while True:
        tmp = os.popen(cmd).read().split()
        if len(tmp) >= 19:
            break
        elif len(tmp) == 2:
            return "no data"
    time_stamp = int(tmp[7].replace("readytime=","").replace("\"",""))
    utc_time = datetime.datetime.utcfromtimestamp(time_stamp)
    build_time = utc_time.strftime("%Y-%m-%d_%H:%M:%S")
    return build_time

def check_and_rebuild(proj, pkg_dict, failed_pkglist, arch, ip, pwd):
    """
    According to the condition, check failed pkgs and rebuild
    """
    url = "http://117.78.1.88:82/obs_build_log/%s/%s" % (proj, arch)
    for pkg in failed_pkglist:
        if pkg not in pkg_dict:
            pkg_dict[pkg] = [0, 'NA']
        if pkg_dict[pkg][0] < 3:
            build_time = get_last_build_time(proj, arch, pkg)
            if build_time == "no data":
                log.info("%s %s %s get build time failed!" % (proj, arch, pkg))
                continue
            if pkg_dict[pkg][1] != build_time:
                log.info("%s %s %s build time old is %s, new is %s." % (proj, arch, pkg, pkg_dict[pkg][1], build_time))
                pkg_dict[pkg][1] = build_time
                download_upload_build_log(proj, arch, pkg, build_time, ip, pwd, "failed_status")
                log_file = os.path.join(proj, arch, pkg, "%s.log" % build_time)
                error_type = classify_pkg_error_type(log_file)
                tmp = build_time + "_" + error_type + ".log"
                log_url = os.path.join(url, pkg, tmp)
                add_comment_msg(proj, pkg, log_url)
                if error_type == "Network_problem":
                    cmd = "osc rebuild %s %s standard_%s %s &>/dev/null" % (proj, pkg, arch, arch)
                    res = os.system(cmd)
                    if res == 0:
                        log.info("rebuild %s %s standard_%s %s success." % (proj, pkg, arch, arch))
                        pkg_dict[pkg][0] += 1
                    else:
                        log.warning("rebuild %s %s standard_%s %s failed." % (proj, pkg, arch, arch))
                else:
                    log.info("%s %s build failed type is %s, no need to rebuild." % (proj, pkg, error_type))
            else:
                log.info("%s %s %s build time old is %s, new is %s, no change." % (proj, arch, pkg, pkg_dict[pkg][1], build_time))
        elif pkg_dict[pkg][0] >= 3:
            build_time = get_last_build_time(proj, arch, pkg)
            if build_time == "no data":
                log.info("%s %s %s get build time failed!" % (proj, arch, pkg))
                continue
            if pkg_dict[pkg][1] != build_time:
                download_upload_build_log(proj, arch, pkg, build_time, ip, pwd, "failed_status")
                pkg_dict[pkg][1] = build_time
            else:
                log.info("%s %s %s build time old is %s, new is %s, no change." % (proj, arch, pkg, pkg_dict[pkg][1], build_time))

def classify_pkg_error_type(log_file_path):
    cmdlist = {"Network_problem": "grep -E \"Name or service not known|connect to backend.openeuler.org\" %s 2>/dev/null" % log_file_path,
            "Code_problem": "grep -E \"make: \*\*\* \[Makefile:|Failures:|BUILD FAILED|Hunk #. FAILED at\" %s 2>/dev/null" % log_file_path,
            "File_conflict_problem": "grep \"conflicts with file from\" %s 2>/dev/null" % log_file_path,
            "Binary_repo_problem": "grep \"getbinaries: missing packages:\" %s 2>/dev/null" % log_file_path,
            "Insufficient_disk_space": "grep -E \"ran out of disk space|syswrite:\" %s 2>/dev/null" % log_file_path
            }
    for err_type,cmd in cmdlist.items():
        res = os.popen(cmd).read()
        if res:
            return err_type
    return "Unknown_problem"

def add_comment_msg(proj, pkg, msg):
    cmd = "osc comment create -c \"%s\" package %s %s &>/dev/null" % (msg, proj, pkg)
    res = os.system(cmd)
    if res == 0:
        log.info("add %s %s comments success." % (proj, pkg))
    else:
        log.info("add %s %s comments failed." % (proj, pkg))

def reset_trigger_times(proj, pkg_dict, arch):
    for key,value in pkg_dict.items():
        if int(value[0]) > 0:
            num = int(value[0]) + 1
            cmd = "osc api -X GET \"build/%s/standard_%s/%s/_jobhistory?package=%s&limit=%s\" 2>/dev/null | grep succeeded" % (proj, arch, arch, key, num)
            tmp = exec_command(cmd, 10)
            if tmp == "TIME_OUT":
                log.info("exec command timeout!")
                continue
            elif len(tmp) > 0:
                value[0] = 0

def monitor_project(proj, ip, pwd):
    pkg_arch_dict = {}
    pkg_list = os.popen("osc ls %s 2>/dev/null" % proj).read().split()
    for pkg in pkg_list:
        pkg_arch_dict.setdefault(pkg, []).append(0)
        pkg_arch_dict.setdefault(pkg, []).append("NA")
    pkg_x86_dict = deepcopy(pkg_arch_dict)
    while True:
        log.info("====================aarch64======================")
        count = 5
        while count:
            reset_trigger_times(proj, pkg_arch_dict, "aarch64")
            cmd = "osc r --csv %s -r standard_aarch64 -a aarch64 2>/dev/null | grep -v 'aarch64' | grep 'failed' \
                    | awk -F ';' '{print $1}'" % proj
            failed_aar_pkgs = exec_command(cmd, 10)
            if failed_aar_pkgs == "TIME_OUT":
                log.info("search aarch64 failed pkgs timeout!")
                count = count - 1
                continue
            else:
                log.info("failed_aar_pkgs:%s" % failed_aar_pkgs)
                check_and_rebuild(proj, pkg_arch_dict, failed_aar_pkgs, "aarch64", ip, pwd)
                break
        log.info("=====================x86_64======================")
        count = 5
        while count:
            reset_trigger_times(proj, pkg_x86_dict, "x86_64")
            cmd = "osc r --csv %s -r standard_x86_64 -a x86_64 2>/dev/null | grep -v 'x86_64' | grep 'failed' \
                    | awk -F ';' '{print $1}'" % proj
            failed_x86_pkgs = exec_command(cmd, 10)
            if failed_x86_pkgs == "TIME_OUT":
                log.info("search x86_64 failed pkgs timeout!")
                count = count - 1
                continue
            else:
                log.info("failed_x86_pkgs:%s" % failed_x86_pkgs)
                check_and_rebuild(proj, pkg_x86_dict, failed_x86_pkgs, "x86_64", ip, pwd)
                break
        log.info("sleep 5 minutes...")
        time.sleep(300)

def access_to_log(proj, arch, pkg_dict, ip, pwd):
    for pkgname,build_time in pkg_dict.items():
        build_time = get_last_build_time(proj, arch, pkgname)
        if build_time == "no data":
            log.info("%s %s %s get build time failed!" % (proj, arch, pkgname))
            continue
        if pkg_dict[pkgname] != build_time:
            log.info("%s %s %s build time old is %s, new is %s." % (proj, arch, pkgname, pkg_dict[pkgname], build_time))
            download_upload_build_log(proj, arch, pkgname, build_time, ip, pwd, "all_status")
            pkg_dict[pkgname] = build_time
        else:
            log.info("%s %s %s build time old is %s, new is %s, no change." % (proj, arch, pkgname, pkg_dict[pkgname], build_time))

def monitor_project_all_pkg(proj, ip, pwd):
    pkg_arch_dict = {}
    pkg_list = os.popen("osc ls %s 2>/dev/null" % proj).read().split()
    for pkg in pkg_list:
        pkg_arch_dict[pkg] = "NA"
    pkg_x86_dict = deepcopy(pkg_arch_dict)
    arch_list = ["aarch64", "x86_64"]
    while True:
        for arch in arch_list:
            log.info("=====================%s======================" % arch)
            if arch == "aarch64":
                access_to_log(proj, arch, pkg_arch_dict, ip, pwd)
            if arch == "x86_64":
                access_to_log(proj, arch, pkg_x86_dict, ip, pwd)
        log.info("sleep 5 minutes...")
        time.sleep(300)


if __name__ == "__main__":
    args_list = sys.argv
    log = get_logger()
    if len(sys.argv) != 5:
        log.info("Please check parameters...")
        sys.exit(0)
    elif args_list[4] == "failed_status":
        monitor_project(args_list[1], args_list[2], args_list[3])
    elif args_list[4] == "all_status":
        monitor_project_all_pkg(args_list[1], args_list[2], args_list[3])
    else:
        log.info("Please check parameters...")

